package data.dictionary.builder;

import java.util.HashMap;
import java.util.Iterator;
import java.util.StringTokenizer;

import data.dictionary.Dictionary_Memory;
import data.preprocessing.StringProcessor;
import data.set.OfflineSet;

import tools.Log;
import tools.Pair;

public class DictionaryBuilderFromOfflineSet {

	private OfflineSet<String> c;
	private StringProcessor sp; // dans le cas ou il s'agit de String !
	

	public DictionaryBuilderFromOfflineSet(OfflineSet<String> c, StringProcessor sp) {
		super();
		this.c = c;
		this.sp = sp;
	}
	
	
	public Dictionary_Memory<String> build() {

		
		Dictionary_Memory<String> dico = new Dictionary_Memory<String>();
		
		Iterator<String> iterDocument = c.iterator();

		int cpt = 0;
		//System.out.println("Construction du dictionnaire a partir d'un corpus");
		while (iterDocument.hasNext()) {

			String doc = iterDocument.next();
			String docPP = sp.map(doc); // pre-processing document

			StringTokenizer st = new StringTokenizer(docPP, " "); // decoupage

			//System.out.println(docPP);
			
			while (st.hasMoreTokens()){
				String mot = st.nextToken();
				//System.out.println(mot);
				if(mot != null)
					dico.addValue(mot);
			}
			cpt++;
			

			if (cpt % 1000 == 0)
				Log.sendMessage("info", this.getClass().getName(),cpt + "docs processed");


		}
		Log.sendMessage("info", this.getClass().getName(),"dico built successfully");
		
		return dico;
	}

	
}
